{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "8d409bbc-8282-48b7-9dab-8945d0d7151f",
   "metadata": {},
   "outputs": [],
   "source": [
    "from dotenv import load_dotenv\n",
    "load_dotenv()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "4df61521",
   "metadata": {},
   "outputs": [],
   "source": [
    "from llama_index.core.node_parser import TokenTextSplitter\n",
    "\n",
    "\n",
    "doc = Document(\n",
    "    text=(\"\"\"\n",
    "    ### 第七条 事假  \n",
    "    1. 员工因私事必须本人处理的，可申请事假。  \n",
    "    2. 事假需提前申请并获直属主管批准，紧急情况可事后补办手续。  \n",
    "    3. 事假为无薪假，按日扣除相应工资。  \n",
    "    4. 每月事假原则上不超过3天，全年累计不超过15天，特殊情况需经人力资源部及公司领导审批。\n",
    "    \"\"\"\n",
    "    ),\n",
    "    metadata={\"title\": \"Vacation Questions\"}\n",
    ")\n",
    "\n",
    "# Token 切片\n",
    "\n",
    "splitter = TokenTextSplitter(\n",
    "    chunk_size=32,\n",
    "    chunk_overlap=4,\n",
    "    separator=\"\\n\"\n",
    ")\n",
    "\n",
    "nodes = splitter.get_nodes_from_documents([doc])\n",
    "\n",
    "for node in nodes:\n",
    "    print(node.text)\n",
    "    print(node.metadata)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "42d91844",
   "metadata": {},
   "outputs": [],
   "source": [
    "# 句子切片\n",
    "sentence_splitter = SentenceSplitter(\n",
    "    chunk_size=512,\n",
    "    chunk_overlap=50\n",
    ")\n",
    "evaluate_splitter(sentence_splitter, documents, question, ground_truth, \"Sentence\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "3f9b58fa",
   "metadata": {},
   "outputs": [],
   "source": [
    "# 句子窗口切片\n",
    "sentence_window_splitter = SentenceWindowNodeParser.from_defaults(\n",
    "    window_size=3,\n",
    "    window_metadata_key=\"window\",\n",
    "    original_text_metadata_key=\"original_text\"\n",
    ")\n",
    "# 注意：句子窗口切片需要特殊的后处理器\n",
    "query_engine = index.as_query_engine(\n",
    "    similarity_top_k=5,\n",
    "    streaming=True,\n",
    "    node_postprocessors=[MetadataReplacementPostProcessor(target_metadata_key=\"window\")]\n",
    ")\n",
    "evaluate_splitter(sentence_window_splitter, documents, question, ground_truth, \"Sentence Window\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "967008d9",
   "metadata": {},
   "outputs": [],
   "source": [
    "# 语义切片\n",
    "\n",
    "semantic_splitter = SemanticSplitterNodeParser(\n",
    "    buffer_size=1,\n",
    "    breakpoint_percentile_threshold=95,\n",
    "    embed_model=Settings.embed_model\n",
    ")\n",
    "evaluate_splitter(semantic_splitter, documents, question, ground_truth, \"Semantic\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "67e7896b",
   "metadata": {},
   "outputs": [],
   "source": [
    "# markdown 切片\n",
    "markdown_splitter = MarkdownNodeParser()\n",
    "evaluate_splitter(markdown_splitter, documents, question, ground_truth, \"Markdown\")"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "AI工程化(week03)",
   "language": "python",
   "name": "week03"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.11.13"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}
